import string
import c_stemming as stemming
import indeks
import node

file_with_morfo = 'morfo3.txt'
file_with_wiki = 'wiki2.txt'
output_file = 'wiki3.txt'

morfo = open(file_with_morfo, 'r')
wiki = open(file_with_wiki, 'r')
out = open(output_file, 'w')

print "Przeprowadzam steming na pliku " + file_with_wiki + ' z uzyciem morfo ' + file_with_morfo
print "Rezultat zostanie zapisany w " + output_file


article = 1
position = 0
val = 0


morfBases = {}
for line in morfo:
   tokens = line.split()
   node.setVal( tokens[0] )
   morfBases[ tokens[0] ] = tuple( tokens[1:] ) 

print "Wczytalem caly plik " + file_with_morfo

for line in wiki:
   tokens = line.split()
   for token in tokens:
      position += 1      
      if token[0] == '#':
         article += 1
         position = 1
         if article % 10000 == 0:
            print article
         out.write(token + ' ')
      elif token in morfBases:
         for base in morfBases[token][:-1]:
            val = node.setVal(base)
            out.write(base+'|')
            indeks.mock_put(val, article, position)
         base = morfBases[token][-1]
         val = node.setVal(base)
         out.write(base)
         indeks.mock_put(val, article, position)
         out.write(' ')        
      else:
         sToken = stemming.stemmToken(token) 
         if sToken is not None:
            out.write(sToken + ' ')
            temp = node.setVal(sToken)
            indeks.mock_put(temp, article, position)            
         else:
            temp = node.setVal(token)
            indeks.mock_put(temp, article, position)
            out.write(token + ' ')
    
   out.write('\n')

out.close()
indeks.indeks_serialize("indeksStaticStem.dat")
node.node_serialize("nodeStaticStem.dat")
